import re

import pandas as pd
from opencc import OpenCC
import jieba.posseg as psg
class clean_data():
    # def __init__(self,file_path):
    #     self.data = pd.read_csv(file_path)
    def tradition_tcc(self,contents):
        cc = OpenCC('t2s')
        new_contents = cc.convert(str(contents))
        return new_contents

    def speech_filter(self, email):
        allow_pos = ['n', 'nr', 'ns', 'nt', 'v', 'a']

        email = str(email)
            # 先过滤非中文字符
        email = "".join(re.findall(r'[\u4e00-\u9fff]+', email))
        words = []
        for word, pos in psg.cut(email):
             if pos in allow_pos:
                words.append(word)
        return words



    def remove_null(self, contents, labels):
        new_contents, new_labels = [], []

        for content, label in zip(contents, labels):
            if not content:  # 空字符串或 None 都会跳过
                continue
            new_contents.append(content)
            new_labels.append(label)

        return pd.DataFrame({'contents': new_contents, 'labels': new_labels})

    def save_csv(self,df,file_path):
        df.to_csv(file_path)

# a = clean_data('./data/01初始训练集.csv')
# data = a.data
# contents,labels = data['x'],data['y']
# contents = a.tradition_tcc(contents)
# contents = a.speech_filter(contents)
# df = a.remove_null(contents,labels)
# a.save_csv(df,'./data/03训练集.csv')

